import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;

public class SpiderApplication {
    
    public static void main(String[] args) {
        spidder();
    }
    
    private static void spidder () {
        String rootURL = "https://read.douban.com/ebooks/category/all/?sort=top&dcs=book-hot&dcm=douban&dct=read-more";
        String pageURL = "https://read.douban.com/ebooks/category/all/?cat=book&sort=top&start=";
        ArrayList<Book> bookArrayList = new ArrayList<Book>();
    
        Integer pageIndex = 0;
        Document document  = null;
        try {
            
            while (true) {
                String path = pageURL + pageIndex.toString();
                document = Jsoup
                        .connect(path)
                        .header("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
                        .cookie("ue", "fgcui@outlook.com")
                        .cookie("id", "22c53307851c004f||t=1489104235|et=730|cs=002213fd4808e1fa3defd53567")
                        .cookie("dbcl2", "\"122772708:7P1KrbuJjlI\"")
                        .timeout(500000).get();
    
                Elements elements = document.select("li.store-item");
                
                if (elements.size() == 0) break;
                
                for (Element element: elements) {
                    Element bookElement = element.select("div.title > a").get(0);
                    String bookURL = "https://read.douban.com" + bookElement.attr("href");
                    String author = element.select("span.labeled-text > a").get(0).html();
    
                    Book book = new Book(bookElement.html(), author, bookURL);
    
                    bookArrayList.add(book);
                }
                System.out.println( bookArrayList.size());
                pageIndex+=20;
            }
    
            System.out.println( bookArrayList.size());
            for (Book book: bookArrayList) {
                
                System.out.println(book.getName() + "-----------" + book.getAuthor() + "------------" + book.getBookUrl());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    
        
    }
    
    
    
    
}
